''' <summary>
''' Class to provide very fast importing of delimited text files
''' </summary>
Public Class clsDataText
    Inherits clsData

    Protected sr As IO.StreamReader
    Protected mFileLength As Integer
    Protected mLastLineRead As String
    Protected cBufferSize As Integer = 10000
    Protected mDelimiters As String

    Dim WithEvents csv As LumenWorks.Framework.IO.Csv.CsvReader
    Private mIsCSV As Boolean
    Private mIsTabular As Boolean

    ''' <summary>
    ''' Instantiate clsDataText
    ''' </summary>
    ''' <param name="_Filename">Name of text file to read</param>
    ''' <param name="_Delimiters">String containing all allowable delimiters</param>
    ''' <param name="_HasHeaders">True if first row contains column headers for field names</param>
    ''' <param name="_IsTabular">True if the text file is tabular, and you can use fast CSV reader</param>
    ''' <remarks>CSVReader is unable to use multiple delimiters, use multiple spaces as delimiters, or read non-tabular text files</remarks>
    Public Overloads Sub Initialize(ByVal _Filename As String, ByVal _Delimiters As String, ByVal _HasHeaders As Boolean, ByVal _IsTabular As Boolean)
        MyBase.Initialize(_Filename)
        mDataType = enumDataType.Text
        Open(_Filename, _Delimiters, _HasHeaders, _IsTabular)
    End Sub

    Public Overrides Sub Initialize(ByVal _Filename As String)
        Dim delim As String = ""
        Select Case IO.Path.GetExtension(_Filename).ToUpper
            Case ".TXT" : delim = vbTab
            Case ".CSV" : delim = ","
            Case Else
                delim = " "
        End Select
        mDataType = enumDataType.Text
        Open(_Filename, delim, True, True)
    End Sub

    ''' <summary>
    ''' Instantiate and figure out if can use very fast CsvReader class
    ''' </summary>
    ''' <param name="_Filename">Name of text file to read</param>
    ''' <param name="_Delimiters">String containing all allowable delimiters</param>
    ''' <param name="_HasHeaders">True if first row contains column headers for field names</param>
    ''' <param name="_IsTabular">True if the text file is tabular, and you can use fast CSV reader</param>
    Private Sub Open(ByVal _Filename As String, ByVal _Delimiters As String, ByVal _HasHeaders As Boolean, ByVal _IsTabular As Boolean, Optional ByVal RewindOnly As Boolean = False)
        Try
            If Not RewindOnly Then MyBase.Initialize(_Filename)
            mDelimiters = _Delimiters
            mHasHeaders = _HasHeaders
            mIsTabular = _IsTabular
            If sr IsNot Nothing Then sr.Close() : sr.Dispose()
            sr = New IO.StreamReader(mFilename, System.Text.Encoding.ASCII, False, cBufferSize)
            If _IsTabular AndAlso mDelimiters IsNot Nothing AndAlso mDelimiters.Length = 1 Then
                mIsCSV = True
                If csv IsNot Nothing Then csv.Dispose()
                csv = New LumenWorks.Framework.IO.Csv.CsvReader(sr, mHasHeaders, mDelimiters, """", "\", "#", True, cBufferSize)
                csv.DefaultParseErrorAction = LumenWorks.Framework.IO.Csv.ParseErrorAction.RaiseEvent
            End If
            mFileLength = sr.BaseStream.Length

            'get fieldnames from header (or leave empty)
            mFieldNames.Clear()
            If mHasHeaders Then
                If mIsCSV Then
                    Try
                        For Each s As String In csv.GetFieldHeaders
                            mFieldNames.Add(s)
                        Next
                    Catch ex As Exception
                        'there might be duplicate fieldnames in header, which causes CSV to barf--roll back to non-CSV method (which might be slower)
                        mIsCSV = False
                        csv.Dispose()
                        sr.Close()
                        sr.Dispose()
                        sr = New IO.StreamReader(mFilename, System.Text.Encoding.ASCII, False, cBufferSize)
                        If Read() Then
                            For Each s As String In mItems.ToArray
                                mFieldNames.Add(s)
                            Next
                        End If
                    End Try
                Else
                    If Read() Then
                        For Each s As String In mItems.ToArray
                            mFieldNames.Add(s)
                        Next
                    End If
                End If
            Else 'read first line and assume that all rows have the same number of columns
                Dim NumFields As Integer
                If mIsCSV Then
                    NumFields = csv.FieldCount
                Else
                    If Read() Then NumFields = mItems.Count
                    sr.DiscardBufferedData()
                    sr.BaseStream.Seek(0, IO.SeekOrigin.Begin)
                End If
                For i As Integer = 0 To NumFields - 1
                    mFieldNames.Add("")
                Next
            End If
        Catch ex As Exception
            ErrorMsg(, ex)
        End Try
    End Sub

    ''' <summary>
    ''' Call this when the delimiters or header flag may have changed
    ''' </summary>
    ''' <param name="_Delimiters">String containing all delimiters</param>
    ''' <param name="_HasHeaders">True if first line contains column headers</param>
    Public Overridable Sub Reopen(ByVal _Delimiters As String, ByVal _HasHeaders As Boolean)
        If _Delimiters <> mDelimiters Or _HasHeaders <> mHasHeaders Then Open(mFilename, _Delimiters, _HasHeaders, mIsTabular)
    End Sub

    ''' <summary>
    ''' Call this when need to write to already open file
    ''' </summary>
    Public Overrides Sub Close()
        If csv IsNot Nothing Then csv.Dispose()
        If sr IsNot Nothing Then sr.Close() : sr.Dispose()
    End Sub

    Public Overrides Function StationIDs() As System.Collections.Generic.List(Of String)
        If mStationIDs.Count = 0 Then
            'read all records and get list of unique station ids
            Dim iSta As Integer = FieldIndex("STATION_ID")
            If iSta <> -1 Then
                Rewind()
                While Read()
                    If iSta <> -1 AndAlso mItems(iSta) <> "" AndAlso Not mStationIDs.Contains(mItems(iSta)) Then mStationIDs.Add(mItems(iSta))
                End While
            End If
            mStationIDs.Sort()
            Rewind()
        End If
        Return mStationIDs
    End Function

    Public Overrides Function PCodes() As System.Collections.Generic.List(Of String)
        If mPCodes.Count = 0 Then
            'read all records and get list of unique station ids
            Dim iPcd As Integer = FieldIndex("PCODE")
            If iPcd <> -1 Then
                Rewind()
                While Read()
                    If iPcd <> -1 AndAlso mItems(iPcd) <> "" AndAlso Not mPCodes.Contains(mItems(iPcd)) Then mPCodes.Add(mItems(iPcd))
                End While
            Else 'just return field names
                For Each s As String In mFieldNames
                    If Not IsWRDBField(s) Then mPCodes.Add(s)
                Next
            End If
            Rewind()
        End If
        Return MyBase.PCodes
    End Function

    ''' <summary>
    ''' Read next line from file into memory
    ''' </summary>
    ''' <returns>True if successful</returns>
    Public Overrides Function Read() As Boolean
        Try
            If mIsCSV Then
                If Not MyBase.Read() Then Return False
                'Debug.Print(mRecordNum)
errloop:
                Try
                    csv.DefaultParseErrorAction = LumenWorks.Framework.IO.Csv.ParseErrorAction.RaiseEvent
                    If csv.ReadNextRecord() Then
                        mItems.Clear()
                        For i As Integer = 0 To csv.FieldCount - 1
                            mItems.Add(csv.Item(i))
                        Next
                        Return True
                    Else
                        Return False
                    End If
                Catch ex As Exception
                    'will capture problems in event handler
                    'GoTo errloop
                    Return False
                End Try
            Else
                If sr.EndOfStream Then Return False

                'skip blank and comment lines beginning with ; or #
                Do
                    mRecordNum += 1
                    mLastLineRead = sr.ReadLine
                    'Debug.Print(mLastLineRead)
                Loop Until sr.EndOfStream OrElse (mLastLineRead.Trim <> "" AndAlso Not (mLastLineRead.StartsWith(";") Or mLastLineRead.StartsWith("#")))
                If mLastLineRead.Trim = "" OrElse mLastLineRead.StartsWith(";") OrElse mLastLineRead.StartsWith("#") Then Return False 'must have encountered EOF before found good line

                Dim CurrentLine As String = mLastLineRead.Trim(" ")
                For Each c As Char In mDelimiters.ToCharArray
                    If c = " " Then
                        'if using space as delimiter, want multiple consecutive spaces to act as a single delimiter
                        Do
                            CurrentLine = CurrentLine.Replace("  ", " ")
                        Loop Until Not CurrentLine.Contains("  ")
                    End If
                Next
                mItems.Clear()
                For Each s As String In CurrentLine.Split(mDelimiters.ToCharArray)
                    s = s.Trim
                    If s = "/" Or s = "\" Then s = "" 'these are special delimiters to help count commas

                    'remove quotes surrounding entry (NOTE: will not correctly parse lines with quoted fields containing commas)
                    If s.Length >= 2 AndAlso s.StartsWith("""") And s.EndsWith("""") Then s = s.Substring(1, s.Length - 2)

                    'don't allow any interior quotes--change to tick marks
                    s = s.Replace("""", "'")
                    mItems.Add(s)
                Next
                Return True
            End If
        Catch ex As Exception
            ErrorMsg(, ex)
            Return False
        End Try

    End Function

    Public Overrides ReadOnly Property MinDate() As Date
        Get
            Rewind()
            If Read() Then
                Return CDate(Items("Date_Time"))
            Else
                Return MyBase.MinDate
            End If
        End Get
    End Property

    Public Overrides ReadOnly Property MaxDate() As Date
        Get
            Rewind()
            If Read() Then
                Dim lastdate As Date
                With csv
                    While .ReadNextRecord
                        lastdate = .Item("Date_Time")
                    End While
                End With
                Return lastdate
            Else
                Return MyBase.MaxDate
            End If
        End Get
    End Property

    Public Overrides Sub Rewind()
        MyBase.Rewind()
        Open(mFilename, mDelimiters, mHasHeaders, mIsTabular, True)
    End Sub

    ''' <summary>
    ''' Get data table containing time series data for a specified station id and pcode
    ''' Table will contain date-time field and result field
    ''' </summary>
    Public Overloads Function GetDataTable(ByVal StationID As String, ByVal PCode As String) As DataTable
        If Not StationIDs.Contains(StationID) Then
            Throw New ApplicationException("Text file does not contain the station: " & StationID)
            Return Nothing
        End If
        Dim itemNum As Integer = -1
        Dim staNum As Integer = -1
        Dim pcodeNum As Integer = -1
        If ContainsFieldName(PCode) Then
            itemNum = FieldIndex(PCode)
        ElseIf ContainsFieldName("RESULT") And ContainsFieldName("PCODE") Then
            itemNum = FieldIndex("RESULT")
            pcodeNum = FieldIndex("PCODE")
            staNum = FieldIndex("STATION_ID")
        Else
            Throw New ApplicationException("Text file does not contain the column: " & PCode)
            Return Nothing
        End If
        Dim dtNum As Integer = -1
        If ContainsFieldName("DATE_TIME") Then
            dtNum = FieldIndex("DATE_TIME")
        Else
            Throw New ApplicationException("Text file does not contain the column: DATE_TIME")
            Return Nothing
        End If
        Dim dt As DataTable = MyBase.SetupDataTable(enumDataTableColumns.Date_Result)
        Rewind()
        While Read()
            If pcodeNum = -1 Then 'data column names are pcodes so get directly
                dt.Rows.Add(mItems(dtNum), mItems(itemNum))
            Else 'normal WRDB fields, so filter by pcode
                If mItems(pcodeNum) = PCode And mItems(staNum) = StationID Then dt.Rows.Add(mItems(dtNum), mItems(itemNum))
            End If
        End While
        Rewind()
        Return dt
    End Function

    Public Overrides Function PercentComplete() As Double
        Return CInt(sr.BaseStream.Position * 100.0 / mFileLength)
    End Function

    Private disposed As Boolean = False

    Protected Overrides Sub Dispose(ByVal disposing As Boolean)
        If Not Me.disposed Then
            If disposing Then
                If sr IsNot Nothing Then
                    sr.Close()
                    sr.Dispose()
                    sr = Nothing
                End If
                If csv IsNot Nothing Then
                    csv.Dispose()
                    csv = Nothing
                End If
            End If
        End If
        Me.disposed = True
    End Sub

    Private Sub csv_ParseError(ByVal sender As Object, ByVal e As LumenWorks.Framework.IO.Csv.ParseErrorEventArgs) Handles csv.ParseError
        With e.Error
            'find CR or CRLF prior to error
            Dim StartPos, EndPos As Integer
            For i As Integer = .CurrentPosition - 3 To 0 Step -1
                If .RawData.Substring(i, 1) = vbCr Or .RawData.Substring(i, 1) = vbLf Then StartPos = i + 1 : Exit For
            Next
            For i As Integer = StartPos To .RawData.Length - 1
                If .RawData.Substring(i, 1) = vbCr Or .RawData.Substring(i, 1) = vbLf Then EndPos = i - 1 : Exit For
            Next
            ErrorMsg(String.Format("Unable to parse import file; error occurred on line {0} and field number {1};\n" &
                                   "you can open the file in Excel or a text editor that displays line numbers\n" &
                                   "and go to the line which is one larger than the record number (headers are not included);\n\n" &
                                   "The line at which the error occurred, and position of the error on that line (^), follows:\n\n{4}^{5}\n\n" &
                                   "The the current buffer being examined at the time of the error is displayed below;\n" &
                                   "the parsing error occurred at position {2} in that buffer:\n\n{3}",
                                   .CurrentRecordIndex + 1, .CurrentFieldIndex + 1, .CurrentPosition, .RawData,
                                   .RawData.Substring(StartPos, Math.Min(EndPos, .CurrentPosition) - StartPos + 1).Replace(vbCr, "").Replace(vbLf, ""),
                                   .RawData.Substring(.CurrentPosition, Math.Max(0, EndPos - .CurrentPosition + 1))))
            e.Action = LumenWorks.Framework.IO.Csv.ParseErrorAction.ThrowException
        End With
    End Sub
End Class
